#!/bin/sh

# Copyright 2003, 2004, 2005, 2006, 2007, 2008 Grid Technology Research Center,
# National Institute of Advanced Industrial Science and Technology.
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#     http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#
# usage exmaple:
#  ./sge_rsvq_start_rsv2.sh R123 1 rsvHostA rsvHostB
#
# PHASE
#	0 : resume preQ
#	1 : delete preQ, resume mainQ
#	2 : resume mainQ
#	other: ignore

if [ "$#" -lt 2 ]; then
  echo 'usage: sge_rsvq_start_rsv2.sh RSVID PHASE ["RSVHOST_A [RSVHOST_B ...]"]'
  exit 1
fi

RSVID=$1

SGE_QNAME="${RSVID}"
SGE_HOSTLIST="@${SGE_QNAME}_hosts"
SGE_PHASE=$2
shift 2
SGE_RSVHOSTS=$1
SGE_PREQNAME="${RSVID}pre"

rerun_unrsv_jobs() {
	#
	# Suspend unreserved queue instances on reserved host
	#
	for HOST in $SGE_RSVHOSTS; do
	  # get queue instances on $HOST
	  #   excluding reserve queue which name starts with 'R'.
	  QINSTANES_ON_HOST=`qstat -f | grep ${HOST} | grep -v '^R' | awk '{print $1}'`

	  # Suspend thier queue instances.
	  # NOTE: jobs on the queue instance is marked as suspended
	  for QINS_NAME in $QINSTANES_ON_HOST; do
	    qmod -sq ${QINS_NAME}
	  done
	done

	#
	# Rerun jobs on thier queue instances
	#
	for HOST in $SGE_RSVHOSTS; do
	  # get suspended jobid and its owner on queue instance on $HOST
	  #   excluding reserve queue which name starts with 'R'.
	  JOBS_ON_HOST=`qstat -q [^R]*@${HOST} -s s | grep '^ ' | awk '{print $1" "$4}'`

	  # To rerun unreserved job, we do
	  #   1) suspend job (suspending queue makes jobs suspended actually),
	  #   2) resubmit (copy and submit) job by job owner's account,
	  #   3) delete job which suspended by 1).
	  i=0
	  for ENTRY in $JOBS_ON_HOST; do
	    t=`expr $i % 2`
	    if [ $t -eq 0 ]; then
	      JOBID=$ENTRY
	    else
	      OWNER=$ENTRY
	      qmod -sj $JOBID
	      sudo -u $OWNER sh -c "source $SGE_ROOT/default/common/settings.sh; qresub $JOBID"
	      qdel $JOBID
	    fi
	    i=`expr $i + 1`
	  done
	done
}

resume_reserve_queue() {
	arg_qname=$1

	# Set final reserved host names (delete unused candidate host names)
	### qconf -rattr hostgroup hostlist ${SGE_RSVHOSTS} ${SGE_HOSTLIST}
	SGE_HOSTFILE="/tmp/sgersv_${SGE_HOSTLIST}"
	echo "group_name ${SGE_HOSTLIST}" > ${SGE_HOSTFILE}
	echo "hostlist ${SGE_RSVHOSTS}" >> ${SGE_HOSTFILE}
	qconf -Mhgrp ${SGE_HOSTFILE}
	if [ $? -ne 0 ]; then
		rm -f ${SGE_HOSTFILE}
		exit 1
	fi
	rm -f ${SGE_HOSTFILE}

	# Resume reserve queue only with reserved hosts
    qmod -usq ${arg_qname}
}

#
# check all host names and reserved host names
#
SGE_ALLHOSTS=`qconf -shgrp ${SGE_HOSTLIST} | grep hostlist | awk -Fhostlist\  '{print $2}'`
if [ -z "$SGE_ALLHOSTS" ]; then
  exit 1
fi

if [ -z "$SGE_RSVHOSTS" ]; then
	SGE_RSVHOSTS=`qconf -shgrp ${SGE_HOSTLIST} | grep hostlist | awk -Fhostlist\  '{print $2}'`
	if [ -z "$SGE_RSVHOSTS" ]; then
	  exit 1
	fi
fi

#
# Resume reserve queue
#
if [ "$SGE_PHASE" -eq "0" ]; then
	rerun_unrsv_jobs
	# resume preQ
	resume_reserve_queue "${SGE_PREQNAME}"
elif [ "$SGE_PHASE" -eq "1" ]; then
	# delete preQ, resume mainQ
	qconf -dq "${SGE_PREQNAME}"
	resume_reserve_queue "${SGE_QNAME}"
elif [ "$SGE_PHASE" -eq "2" ]; then
	rerun_unrsv_jobs
	# resume mainQ
	resume_reserve_queue "${SGE_QNAME}"
fi
